import requests
import json
import lxml.html
import pymongo
# 1. 请求头伪装，尽可能避免被网站的反扒机制识别的
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 Edg/128.0.0.0'}
url = 'https://www.wzj.com/proxy/search/products?limit=10&offset=20&search_page_type=classification&category_id=56,86,159,85,158,249,54,59,152,57,58,263,153,55,154,292&type=category'
url1 = 'https://www.wzj.com/proxy/search/products?limit=100&offset=30'
response = requests.get(url1, headers=headers)
print(response.status_code)
print(response.text)
json_data = json.loads(response.text)  # 将ajax获取到的json字符串转为对象
results = json_data['results']  # 从完整的数据中，提取我们需要的数据
print(results)

client = pymongo.MongoClient('localhost', 27017)
db = client['spider-db']
collection = db['wzj_product']
collection.insert_many(results)

